from selenium import webdriver
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.common.by import By
from time import sleep
from lxml import etree
from pymongo import MongoClient

#无界面模式
options = webdriver.ChromeOptions()
options.add_argument('headless')  # 无界面启动，也可以直接设置options.headless=True
driver = webdriver.Chrome(options=options)   

url='https://item.jd.com/100012404503.html#comment'
#url = input('输入网址：')
driver.get(url)
# 停顿
wait = WebDriverWait(driver, 10)
sleep(3)
global link_page
link_page=0
global dg             
dg=1 
#滑到中间
js = "var q=document.documentElement.scrollTop=5000"  
driver.execute_script(js)
sleep(2)
#刷新防止不加载评论
driver.refresh()
sleep(2)
#点击商品评论
driver.find_element_by_xpath('//*[@id="detail"]/div[1]/ul/li[5]').click()
def main():    
    global link_page
    #输入想要获取到的页数，不超过最大页码数
    link_page=int(input('想要获取到多少页:'))
    ht()

def ht():   
    js = "var q=document.documentElement.scrollTop=4500"  
    driver.execute_script(js)
    sleep(3)
    hty=driver.page_source
    li(hty)
    sleep(4)
        
#想要获取到多少页(进行判断)
def link_clik():
    global dg
    global link_page        
    if dg < link_page:
        dg+=1
        print ("正在获取第%s页的评论信息"%dg)
        link_next()


#详情页商品评论获取
def li(hty):
    pq = etree.HTML(hty)    
    rt=pq.xpath('//div[@class="comment-item"]')
    pl_r=[]
    img_r=[]
    user_r=[]
    start_r=[]
    #解析详情页数据
    for j in range(len(rt)):
        #获取评论
        pinglun=pq.xpath('//div[@class="comment-column J-comment-column"]/p')[j].xpath('string(.)')
        pl_r.append(pinglun)
        #获取用户昵称
        user=pq.xpath('//div[@class="user-info"]')[j].xpath('string(.)').strip(' ')
        user_r.append(user)
        #获取用户头像
        user_img='https:'+pq.xpath('//div[@class="user-info"]/img/@src')[j]
        img_r.append(user_img)
        #获取时间
        time_r=[]
        time1=pq.xpath('//div[@class="order-info"]/span/text()')
        for p in time1:
            if ':'in p:
                time_r.append(p) 
        #获取评价等级
        start=pq.xpath('//div[@class="comment-column J-comment-column"]/div[1]/@class')[j].split(' ')[-1]
        start_r.append(start)
        #获取视频链接
        video1=pq.xpath('//div[@class="comment-column J-comment-column"]/div[4]/@class')
        video=pq.xpath('//div[@class="comment-column J-comment-column"]/div[4]/div[1]/video[@class="vjs-tech"]/@src')
        #获取J-video-view-wrap clearfix 所在的索引
        id1=[i for i,x in enumerate(video1) if x=='J-video-view-wrap clearfix']
        id2=[i for i,x in enumerate(video1) if x!='J-video-view-wrap clearfix']
        for y in range(len(video)):
            video1[id1[y]]=video[y]
        for p in id2:
            video1[p]='None'            
        #获取图片链接
        img=pq.xpath('//div[@class="comment-column J-comment-column"]/div[2]/@class')
        #获取索引
        img_index=[i for i,x in enumerate(img) if x=='pic-list J-pic-list']
        img_index1=[i for i,x in enumerate(img) if x!='pic-list J-pic-list']
        for i in range(len(img_index)):
            bp=pq.xpath('//div[{}]/div[2]/div[2]/a[@class="J-thumb-img"]/img/@src'.format(img_index[i]+1))       
            all=' '
            for qw in range(len(bp)):
                ft='https:'+bp[qw]
                all=all+ft+'\n'
            #替换指定位置
            img[img_index[i]]=all
        for z in img_index1:
            img[z]='None'
    all_in=[]   
    for a,b,c,d,e,f,g in zip( pl_r,user_r,img_r,time_r,start_r,video1,img):
        F={'评价':a,
           '用户名':b,
           '头像':c,
           '时间':d,
           '星级':e,
           '视频链接':f,
           '晒图链接':g}
        all_in.append(F)
        print(all_in)
        print('每页总数为:{}'.format(len(all_in)))
    save_db(all_in)
    link_clik()
 

def save_db(save_data):
    #连接数据库
    client=MongoClient()
    #创建数据库
    database=client['jd_link']
    #创建集合
    db=database['link']
    db.insert(save_data)               

#设置点击下一页按钮
def link_next():
    element = driver.find_element_by_class_name('ui-pager-next')
    driver.execute_script("arguments[0].click();", element)
    sleep(5)
    ht()

main()


